#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
"""
@author: caijj
@version: V1.0
@file: zg_sz_new_house_info.py
@time: 2022/02/11
"""
import multiprocessing
import os
import re
import uuid
from time import sleep
import requests
from fake_useragent import UserAgent
from lxml import etree
from selenium import webdriver
from tools import timeutil
from tools.database_tool import DataBaseTool
from tools.decorator import update_flag

tag = ''


def agent():
    ua = UserAgent(verify_ssl=False)
    headers = {'User-Agent': ua.random}
    return headers


def chrome_enter_target_page(url='https://su.xinfang.zhuge.com/'):  # chrome进入目标页面
    chrome_options = webdriver.ChromeOptions()
    chrome_options.add_argument('--headless')
    chrome_options.add_argument('--disable-gpu')
    web_driver = webdriver.Chrome(options=chrome_options)
    web_driver.maximize_window()
    web_driver.get(url)
    return web_driver


@update_flag('mega_syn_hand', 'mega_zg_sz_xf_estate_info')
def get_new_house_list():
    del_data()
    house_info_list = []
    time = timeutil.dt_strptime("%Y-%m-%d %H%M%S")
    web_driver = chrome_enter_target_page()
    html_source = web_driver.page_source  # 该属性可以获取当前浏览器的当前页的源码（html）
    tree = etree.HTML(html_source)
    total_pages = tree.xpath('//div[@class="list_page_box"]/p[@class="page_rp"]')[0].text[1:-1]
    for i in range(int(total_pages)):
        if i == 0:
            pass
        else:
            page_number = str(i + 1)
            url = 'https://' + 'su.xinfang.zhuge.com' + '/page/' + page_number + '/'
            web_driver = chrome_enter_target_page(url)
            sleep(2)
            html_source = web_driver.page_source
        tree = etree.HTML(html_source)
        detail_ele_list = tree.xpath('//div[@class="list_box"]//li[@class="list_li"]/div[@class="house_details"]')
        for detail_ele in detail_ele_list:
            index = detail_ele_list.index(detail_ele)
            house_id = uuid.uuid4()
            # img_src = tree.xpath("//div[@class='list_box']//li[@class='list_li'][" + str(
            #     index + 1) + "]/a/img/@src")[0]
            detail_link = tree.xpath("//div[@class='list_box']//li[@class='list_li'][" + str(
                index + 1) + "]/div[@class='house_details']/a/@href")[0]
            estate_name = tree.xpath(
                "//div[@class='list_box']//li[@class='list_li'][" + str(index + 1) + "]/div[@class='house_details']/a")[
                0].text
            p_ele_list = tree.xpath(
                "//div[@class='list_box']//li[@class='list_li'][" + str(index + 1) + "]/div[@class='house_details"
                                                                                     "']/p[@class='ellipsis house_p']")
            if len(p_ele_list) == 3:
                estate_address = tree.xpath("//div[@class='list_box']//li[@class='list_li'][" + str(
                    index + 1) + "]/div[@class='house_details']/p[1]")[0].text
                house_type = tree.xpath("//div[@class='list_box']//li[@class='list_li'][" + str(
                    index + 1) + "]/div[@class='house_details']/p[2]")[0].text
                house_type, house_area = house_type.split('——')[0], house_type.split('——')[1]
                opening_time = tree.xpath("//div[@class='list_box']//li[@class='list_li'][" + str(
                    index + 1) + "]/div[@class='house_details']/p[3]")[0].text.split('：')[1]
            else:
                estate_address = tree.xpath("//div[@class='list_box']//li[@class='list_li'][" + str(
                    index + 1) + "]/div[@class='house_details']/p[1]")[0].text
                house_type, house_area = '', ''
                opening_time = tree.xpath("//div[@class='list_box']//li[@class='list_li'][" + str(
                    index + 1) + "]/div[@class='house_details']/p[2]")[0].text.split('：')[1]
            price_ele_list = tree.xpath("//div[@class='list_box']//li[@class='list_li'][" + str(
                index + 1) + "]/div[@class='house_details']/div[@class='price_box']//span")
            location = geocodeB(estate_address)  # 获取经纬度
            if len(estate_address.split('] ')) == 2:
                estate_address, regions = estate_address.split('] ')[1], estate_address.split('] ')[0]
                if len(regions.split('-')) == 2:
                    region, business_district = regions.split('-')[0][1:], regions.split('-')[1]
                else:
                    region, business_district = regions[1:], ''
            else:
                region, business_district = '', ''
            if len(price_ele_list) == 0:
                price = '价格待定'
            else:
                price = price_ele_list[0].text
            tag_ele_list = tree.xpath("//div[@class='list_box']//li[@class='list_li'][" + str(
                index + 1) + "]/div[@class='house_details']/ul[@class='tag-ul']/li")
            for tag_ele in tag_ele_list:
                global tag
                if tag_ele_list.index(tag_ele) == 0:
                    tag = tag_ele.text
                else:
                    tag = tag + ',' + tag_ele.text
            # ditu_url = 'https:' + detail_link + 'ditu/'  # 获取经纬度
            # web_driver2 = chrome_enter_target_page(ditu_url)
            # html_source = web_driver2.page_source  # 该属性可以获取当前浏览器的当前页的源码（html）
            # sleep(1.5)
            # # print(html_source)
            # tree2 = etree.HTML(html_source)
            # lng = tree2.xpath("//input[@id='lng']")[0].get('value')
            # lat = tree2.xpath("//input[@id='lat']")[0].get('value')
            house_info_list.append(
                (house_id, estate_name, estate_address, opening_time, house_type, tag, price, detail_link, time,
                 location[0], location[1], region, business_district, house_area))
            insert_new_house_detail_data(house_info_list)
            house_info_list = []
        web_driver.quit()


def get_new_house_model(model_link_list, count=0):
    web_driver = None
    model_info_list = []
    # model_link_list = get_house_model_link()
    time = timeutil.dt_strptime("%Y-%m-%d %H%M%S")
    try:
        for model_link_info in model_link_list[count:]:
            house_id = model_link_info[0]
            model_link = model_link_info[1]
            url = 'https:' + model_link + 'huxing/'
            web_driver = chrome_enter_target_page(url)
            sleep(3)
            html_source = web_driver.page_source  # 该属性可以获取当前浏览器的当前页的源码（html）
            tree = etree.HTML(html_source)
            total_model = tree.xpath("//ul[@class='huxing_top']/li[@class=' check']")[0].text
            total_model = re.findall(r'[(](.*?)[)]', total_model)[0]
            if int(total_model) > 12:
                web_driver.find_element_by_xpath(
                    "//a[@class='look_more_block j_more_lists' and contains(text(), '点击加载更多')]").click()
                sleep(2)
                html_source = web_driver.page_source  # 该属性可以获取当前浏览器的当前页的源码（html）
                tree = etree.HTML(html_source)
                model_ele_list = tree.xpath("//ul[@class='huxing_block second clearfix']/li[@class='huxing_li']")
            else:
                model_ele_list = tree.xpath("//ul[@class='huxing_block second clearfix']/li[@class='huxing_li']")
            for model_ele in model_ele_list:
                index = model_ele_list.index(model_ele)
                model_id = uuid.uuid4()
                # print(house_id)
                model_name = tree.xpath("//ul[@class='huxing_block second clearfix']/li[@class='huxing_li'][" + str(
                    index + 1) + "]//p[@class='huxing_p']")[0].text
                on_sale_status = tree.xpath("//ul[@class='huxing_block second clearfix']/li[@class='huxing_li'][" + str(
                    index + 1) + "]//i[@class='tag']")[0].text
                area_ele = tree.xpath("//ul[@class='huxing_block second clearfix']/li[@class='huxing_li'][" + str(
                    index + 1) + "]//p[contains(text(),'建面')]")
                if len(area_ele) == 0:
                    area = tree.xpath("//ul[@class='huxing_block second clearfix']/li[@class='huxing_li'][" + str(
                        index + 1) + "]//p[contains(text(),'套内')]")[0].text.split(': ')[1]
                else:
                    area = area_ele[0].text.split(': ')[1]
                price = tree.xpath("//ul[@class='huxing_block second clearfix']/li[@class='huxing_li'][" + str(
                    index + 1) + "]//span[contains(text(),'参考总价')]//following-sibling::b")[0].text
                model_info_list.append((model_id, house_id, model_name, area, price, on_sale_status, time))
            insert_new_house_model_data(model_info_list)
            model_info_list = []
            web_driver.quit()
            count += 1
    except Exception as e:
        if web_driver is not None:
            web_driver.quit()
        print(e)
        sleep(30)
        print('诸葛网户型介绍需手动验证 + 访问地址为%s' % (url))
        get_new_house_model(count)


def geocodeB(address):
    """
    @ address: 名称字符串
    @ 返回值：经度，纬度
    """
    base_url = "http://api.map.baidu.com/geocoder?address={address}&output=json&key=9NFsYXeiyhl6rXnCDQbetjZYW4tLFDo5".\
        format(address=address)
    response = requests.get(base_url)
    answer = response.json()
    latitude = answer['result']['location']['lng']
    longitude = answer['result']['location']['lat']
    return latitude, longitude


def get_house_model_link(process_count, table_name='mega_zg_sz_xf_estate_info'):
    data_list = []
    sql = "select id,link,estate_name from {}".format(table_name)
    db = DataBaseTool()
    result = db.select_sql(sql)
    step = len(result) // process_count
    # remainder = len(result) % process_count
    for i in range(1, process_count+1):
        if i == range(1, process_count+1)[-1]:
            data = result[step*(i-1):]
        else:
            data = result[step*(i-1):step*i]
        data_list.append(data)
    return data_list


def insert_new_house_detail_data(data, table_name='mega_zg_sz_xf_estate_info'):
    db = DataBaseTool()
    sql = "insert into {}(id, estate_name, estate_address, opening_time, house_type, tag, average_price, link, " \
          "create_time, longitude, latitude, region, business_district, house_area) values(%s,%s,%s,%s,%s,%s,%s,%s,%s," \
          "%s,%s,%s,%s,%s)".format(table_name)
    try:
        cursor = db.conn.cursor()
        cursor.executemany(sql, data)
        # cursor.execute(sql, (x, y))
        db.conn.commit()
    except Exception as e:
        print(e)
    db.conn.close()


def insert_new_house_model_data(data, table_name='mega_zg_sz_xf_model_info'):
    db = DataBaseTool()
    sql = "insert into {}(id, house_id, model_name, area, price, on_sale_status, create_time) values(%s,%s,%s,%s,%s," \
          "%s,%s)".format(table_name)
    try:
        cursor = db.conn.cursor()
        cursor.executemany(sql, data)
        # cursor.execute(sql, (x, y))
        db.conn.commit()
    except Exception as e:
        print(e)
    db.conn.close()


def del_data(table_name='mega_zg_sz_xf_estate_info'):
    db = DataBaseTool()
    sql = "delete from {}".format(table_name)
    db.update_sql(sql)


def check_file_is_exists(csv_name):
    if os.path.isfile(csv_name):
        os.remove(csv_name)


@update_flag('mega_syn_hand', 'mega_zg_sz_xf_model_info')
def get_new_house_model_data():
    process_count = multiprocessing.cpu_count()
    pool = multiprocessing.Pool(multiprocessing.cpu_count())

    house_model_link_list = get_house_model_link(process_count)

    pool.map(get_new_house_model, house_model_link_list)
    pool.close()
    pool.join()


if __name__ == '__main__':
    # get_new_house_list()

    del_data('mega_zg_sz_xf_model_info')
    get_new_house_model_data()  # 诸葛网新房户型信息（mega_zg_sz_xf_model_info）
